{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00538969,  1.4048686 ,  0.54590935, -0.26896894, -0.0062386 ,\n",
       "       -0.12365669,  0.        ,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('LunarLander-v2',\n",
    "                       continuous=False,\n",
    "                       render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZ8klEQVR4nO3de1BU9/038Pfeue6usLArkRWa+hQVNQYvrE6eZpRK1aTG6Dw2tZWxmebRYCZWJ9My05qkfxSbzCRtWqO9/GKc36+G1DxjmqjRsXj7paVeCCgXQVTIorKLCuyuCMsu+33+IJyfa7yhsF9W3q+Z7wx7vl/Ofs6B8+Zc9hxUQggBIiIJ1LILIKKRiwFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSSAugTZs2ISMjAzExMZg5cyaOHTsmqxQikkRKAH344YdYt24dXn31VXzxxReYMmUK8vPz0draKqMcIpJEJeNm1JkzZ2L69On4wx/+AAAIhUJIT0/HSy+9hJ///OeRLoeIJNFG+g17enpQXl6OoqIiZZparUZeXh7Kyspu+T1+vx9+v195HQqF0NbWhuTkZKhUqiGvmYgGRggBn8+HtLQ0qNW3P9CKeABduXIFvb29sFqtYdOtVivq6upu+T3FxcV4/fXXI1EeEQ2i5uZmjBkz5rb9EQ+g+1FUVIR169Yprz0eD+x2u8SK6N6okPvYj+HI/r9oaTuJTw/+HLNzVmPKN/4PNGrDPc/l6rUzKNn7PB5N/9948vFXcN51GLsP/QK9vT1DWDsNhsTExDv2RzyALBYLNBoN3G532HS32w2bzXbL7zEYDDAY7v0XloaHZHMmJv2vxdBqY1DV8DE6u65Cp42BQZ8I7QACSKeNA1TAOed/Y0rWUmSlL0BN+qc41/TfQ1g9DYa7nSKJ+FUwvV6PnJwclJaWKtNCoRBKS0vhcDgiXQ4NEa3GgCemFcKSMA7nWw6joekQAKBX9KA70I6uQNtXrV1p3YEOdAf7m0dpgdB1AICv042as7sBhDB7ymrExJjkLSANCimHYOvWrUNBQQGmTZuGGTNm4Le//S06OzuxcuVKGeXQEEgfnYMx1sfRHejAier/RE+gEwBwtHIrTtb9P2g1Bmg0emg1Bmg1emg0hq++vvG1Hhq1Hj3BTvj91wAA9ef3Y8KjC5CW/DjGZT6JqtN/l7iU9KCkBNCyZctw+fJlbNiwAS6XC4899hj27t37tRPTFL2CwQDcV2vRfs2JCy0VyvR2TzPguf/5+jrdqD23G4HQdXR1dwBQAeBjzaOVlM8BPSiv1wuTibvfw51elwCtRofr3e2DOl+dNhZareGrAKLhzOPxwGg03rY/Kq6CUXTqCVxDT2Dw5xsIdiEQ7Br8GVPE8WZUIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEiaAQfQkSNH8PTTTyMtLQ0qlQoff/xxWL8QAhs2bMDo0aMRGxuLvLw8NDQ0hI1pa2vD8uXLYTQaYTab8fzzz+PatWsPtCBEFH0GHECdnZ2YMmUKNm3adMv+N954A++88w62bNmCo0ePIj4+Hvn5+eju7lbGLF++HDU1Ndi/fz927dqFI0eO4IUXXrj/pSCi6CQeAACxc+dO5XUoFBI2m028+eabyrSOjg5hMBjEBx98IIQQora2VgAQx48fV8Z89tlnQqVSiYsXL97T+3o8HgGAjY1tmDePx3PHbXlQzwE1NjbC5XIhLy9PmWYymTBz5kyUlZUBAMrKymA2mzFt2jRlTF5eHtRqNY4ePXrL+fr9fni93rBGRNFvUAPI5XIBAKxWa9h0q9Wq9LlcLqSmpob1a7VaJCUlKWNuVlxcDJPJpLT09PTBLJuIJImKq2BFRUXweDxKa25ull0SEQ2CQQ0gm80GAHC73WHT3W630mez2dDa2hrWHwwG0dbWpoy5mcFggNFoDGtEFP0GNYAyMzNhs9lQWlqqTPN6vTh69CgcDgcAwOFwoKOjA+Xl5cqYAwcOIBQKYebMmYNZDhENdwO46CWEEMLn84mKigpRUVEhAIi33npLVFRUiC+//FIIIcTGjRuF2WwWf//738WpU6fEokWLRGZmpujq6lLm8d3vfldMnTpVHD16VHz++edi3Lhx4rnnnrvnGngVjI0tOtrdroINOIAOHjx4yzcqKCgQQvRdiv/lL38prFarMBgMYu7cuaK+vj5sHlevXhXPPfecSEhIEEajUaxcuVL4fD4GEBvbQ9buFkAqIYRAlPF6vTCZTLLLIKK78Hg8dzxnGxVXwYjo4cQAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikGVAAFRcXY/r06UhMTERqaiqeeeYZ1NfXh43p7u5GYWEhkpOTkZCQgCVLlsDtdoeNcTqdWLhwIeLi4pCamopXXnkFwWDwwZeGiKKLGID8/HyxdetWUV1dLSorK8WCBQuE3W4X165dU8asWrVKpKeni9LSUnHixAmRm5srZs2apfQHg0GRnZ0t8vLyREVFhdizZ4+wWCyiqKjonuvweDwCABsb2zBvHo/njtvygALoZq2trQKAOHz4sBBCiI6ODqHT6cSOHTuUMadPnxYARFlZmRBCiD179gi1Wi1cLpcyZvPmzcJoNAq/339P78sAYmOLjna3AHqgc0AejwcAkJSUBAAoLy9HIBBAXl6eMiYrKwt2ux1lZWUAgLKyMkyaNAlWq1UZk5+fD6/Xi5qamlu+j9/vh9frDWtEFP3uO4BCoRDWrl2L2bNnIzs7GwDgcrmg1+thNpvDxlqtVrhcLmXMjeHT39/fdyvFxcUwmUxKS09Pv9+yiWgYue8AKiwsRHV1NUpKSgaznlsqKiqCx+NRWnNz85C/JxENPe39fNOaNWuwa9cuHDlyBGPGjFGm22w29PT0oKOjI2wvyO12w2azKWOOHTsWNr/+q2T9Y25mMBhgMBjup1QiGs4GctI5FAqJwsJCkZaWJs6cOfO1/v6T0B999JEyra6uTgBfPwntdruVMX/84x+F0WgU3d3d91QHT0KzsUVHG9SrYKtXrxYmk0kcOnRItLS0KO369evKmFWrVgm73S4OHDggTpw4IRwOh3A4HEp//2X4efPmicrKSrF3716RkpLCy/BsbA9hG9QAut2bbN26VRnT1dUlXnzxRTFq1CgRFxcnFi9eLFpaWsLm09TUJObPny9iY2OFxWIR69evF4FAgAHExvaQtbsFkOqrYIkqXq8XJpNJdhlEdBcejwdGo/G2/bwXjIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiae7rXrDh4rXXXsO5c+fQ0NCAxsZG+Hw+dHV1IQo/2kQ0IkX1BxE9Hg/i4+PR1dUFj8eDxsZGnDlzBtXV1aiqqoLT6URLSwuuX7+OUCjEYCKKsLt9EDHqA+jmhetfnFAohI6ODrS2tqKhoQE1NTWoqalBXV0dLly4AK/Xi66uLhnlE40YIy6AbkcIgd7eXvj9frS1taGpqQlnzpzBqVOncPr0aTQ1NaGlpQV+vx+BQGCIl4BoZGAA3caNix0MBpW9pfr6ehw7dgwffPABnE7nYJUc9VQq4NlnAY0GqKgAPB7gyhUgFJJd2d1ZrcDTTwPnzvW19nbA55Nd1cjAALoPoVAITqcTf/nLX7Bt2zZcvHhxxJ8/UquB998HsrKAQADweoGmJuDChb5Acrn6Nu7ubsDvl11tuOxs4D/+AxCir/ZLl4CWFqC6Gjh/vq9duQJ0dQG9vbKrfbgwgB5AKBTC2bNn8ac//Ql//etfb/vM6pGgP4AmTPh6nxB9wdPeDpw+Dbz9dl8gDRfZ2cB77/Utw42E6GteL9DWBuzaBfzXf0XHXl20uNs2GtWX4YeaWq3GuHHj8Jvf/AY//vGPsWXLFpSUlODy5cuyS5Oqf0/C6wVaW4HaWsDpBE6e7NuQb/o/lMNGf+Bcv953CNbQ0Ff3qVN9e3ItLQyfSOMe0AAEg0HU1NTg3XffxY4dO9De3h6x95ZNrQbeessGr9eFEyf6QubMmb49n+F+MXHy5ES8/LJARcU11NcDX37ZFzbd3X1BSkPnrtvoPT+GcBjpfyLi3Z62NhRCoZAIBALi+PHjoqCgQBiNRulPnYtE02g0oqCgQHod99NycnLElClTpNcxEtuQ/mPCkUilUkGr1WLatGn485//jH379uH73/8+EhISZJdGFHUYQA9Ap9MhNzcX27Ztw+7du7F48WLExcXJLosoajCABoFer8cTTzyB7du345NPPsFTTz3FICK6BwygQaJSqRATE4M5c+bgww8/xN/+9jfk5eXxHyoS3QEDaJCpVCrExcVhwYIF+Pjjj1FSUoInn3wSer1edmlEww4DaIioVCrEx8dj0aJF+OSTT7Bt2zY4HA5otfzoFVE/bg1DTKVSITExEcuWLcOCBQvw6aef4h//+AdOnTqFixcvoq2tjTe/0ojFAIoQlUoFo9GIH/zgB1i2bBkCgQBcLhcaGxtx6tQpVFRUoK6uTnmwmt/vH/H3n9HDjwEUYf2fI9JqtcjMzERmZibmzJmDUCiEzs5OtLe348yZM6ipqUFlZSWqqqpw8eJFXLlyBb29vQylYUSj0cBqtcJkMsHtdqOjowOhEXAvh0qlUn6PExMTkZCQgMTERJjNZowZMwY2mw1msxmvvfbaXefFABom1Go1EhMTkZiYCLvdjrlz5yIUCqGnpwdutxtNTU04efIkKisrUVtbi6amJni9XnR3d8sufcTQ6/VISkpCdnY2Zs6ciSeeeALjx4+H2WzGpUuXUF9fj+PHjys/o8uXL6OzszOq/mhotVoYDAbo9XoYDAYkJycjJSUFKSkpsFgseOSRR5CWlobRo0cjOTkZJpMJRqMRiYmJMBgMSjj5fL7BD6DNmzdj8+bNaGpqAgBMnDgRGzZswPz58wEA3d3dWL9+PUpKSuD3+5Gfn493330XVqtVmYfT6cTq1atx8OBBJCQkoKCgAMXFxTw5exOVSgWNRoPY2FhkZGQgIyMDTz75JHp7e9HV1YW2tjY0NDSgtrYW5eXlqK2thdPpRFtbG3p7e0fEX+KhptVqkZqaiqysLMyePRu5ubmYPHkykpOTERMTA5VKpYw1Go3IysrC9773PfT09MDn8+H8+fOorq5WQqmxsRFtbW0IBoMRDyWVSgW1Wg21Wo2EhAQkJyfDYrHAarUqgTJ69GglaCwWC0aNGgWDwQCdTgedTgetVhu2zHd7v3sxoK1+zJgx2LhxI8aNGwchBLZt24ZFixahoqICEydOxE9/+lPs3r0bO3bsgMlkwpo1a/Dss8/in//8JwCgt7cXCxcuhM1mw7/+9S+0tLRgxYoV0Ol0+PWvfz2QUkYsjUaDhIQEJCQkwG63Y86cOQCArq4uXL58GRcuXMC5c+dw9uxZnD9/Ho2NjcojaK9du4ZgMCh5CYYvnU4Hi8WCiRMnwuFwYNasWZg4cSJsNpvyB/JuG5ZKpYLBYIDBYIDFYsH06dOxcuVK+P1+uN1uNDQ0oLy8HBUVFTh16hRaWlrg9XoH7Q+GRqNBfHw84uPjkZSUhPT0dNjtdowdOxYZGRkYO3ascthoNBq/9vGQew2OwfLAd8MnJSXhzTffxNKlS5GSkoLt27dj6dKlAIC6ujqMHz8eZWVlyM3NxWeffYannnoKly5dUvaKtmzZgp/97Ge4fPnyPX9WRtbd8NFEfPUI2kAggM7OTrhcLly6dEl5cP/Zs2dx8eJFXLp0CR6PB4FA4I5X4zQaDX74wx9i27ZtEVyKwZGTk4NgMIiTJ0+GTdfr9bBYLJgwYQIcDgdmz56NiRMnIiUlBXq9fsg2RiEEgsEgurq68OWXX6Kurg7Hjx9HRUUFGhoa0NraCr/ff8tQ6j9/aDAYkJKSouzBZGZm4hvf+AbsdjusVqsSMv17Lmq1OqLhcq/b6H0f9/T29mLHjh3o7OyEw+FAeXk5AoEA8vLylDFZWVmw2+1KAJWVlWHSpElhh2T5+flYvXo1ampqMHXq1Psth25y48nu2NhYWCwWZGdnhz20v7OzEz6fD62trcre0tmzZ3Hu3Dk4nU60t7fD4/Ggp6dH8tIMjv5DqvHjx2P27NlwOBzIzs4e8sC5mUqlUg5rJk2ahOzsbCxduhSBQABtbW04f/48KisrUV5ejpqaGpjNZuUw3G63w263Iy0tDYmJiYiPj0dsbGzYvKPJgAOoqqoKDocD3d3dSEhIwM6dOzFhwgRUVlZCr9fDbDaHjbdarcqTBF0uV1j49Pf3992O3++H/4bnfHq93oGWTV/p/wXVaDQwGo0wGo145JFHMHXqVAghEAqFEAqFlEO6/hPgDQ0N8Pv9WLFiheQlGLjExESMHTsWjz32GMaPH4/U1FTodLphs7H216HX62Gz2WCz2eBwOBAKhdDb2xt2/ma41DxYBhxA3/rWt1BZWQmPx4OPPvoIBQUFOHz48FDUpiguLsbrr78+pO9B/3PiW6PRQKfTwWg04tFHH8WsWbNklzbi3PizeJgN+FYMvV6Pb37zm8jJyUFxcTGmTJmC3/3ud7DZbOjp6UFHR0fYeLfbDZvNBgCw2Wxw3/S8zv7X/WNupaioCB6PR2nNzc0DLZuIhqEHvhcsFArB7/cjJycHOp0OpaWlSl99fT2cTiccDgcAwOFwoKqqCq2trcqY/fv3w2g0YsKtnnb+FYPBoBwu9Dciin4DOgQrKirC/PnzYbfb4fP5sH37dhw6dAj79u2DyWTC888/j3Xr1iEpKQlGoxEvvfQSHA4HcnNzAQDz5s3DhAkT8KMf/QhvvPEGXC4XfvGLX6CwsJCPrSAagQYUQK2trVixYgVaWlpgMpkwefJk7Nu3D9/5zncAAG+//TbUajWWLFkS9kHEfhqNBrt27cLq1avhcDgQHx+PgoIC/OpXvxrcpSKiqMD/ikFEg+5et1E+D4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImm0sgu4H0IIAIDX65VcCRHdSv+22b+t3k5UBtDVq1cBAOnp6ZIrIaI78fl8MJlMt+2PygBKSkoCADidzjsuHIXzer1IT09Hc3MzjEaj7HKiAtfZ/RFCwOfzIS0t7Y7jojKA1Oq+U1cmk4m/FPfBaDRyvQ0Q19nA3cvOAU9CE5E0DCAikiYqA8hgMODVV1+FwWCQXUpU4XobOK6zoaUSd7tORkQ0RKJyD4iIHg4MICKShgFERNIwgIhImqgMoE2bNiEjIwMxMTGYOXMmjh07JrskaYqLizF9+nQkJiYiNTUVzzzzDOrr68PGdHd3o7CwEMnJyUhISMCSJUvgdrvDxjidTixcuBBxcXFITU3FK6+8gmAwGMlFkWbjxo1QqVRYu3atMo3rLEJElCkpKRF6vV689957oqamRvzkJz8RZrNZuN1u2aVJkZ+fL7Zu3Sqqq6tFZWWlWLBggbDb7eLatWvKmFWrVon09HRRWloqTpw4IXJzc8WsWbOU/mAwKLKzs0VeXp6oqKgQe/bsERaLRRQVFclYpIg6duyYyMjIEJMnTxYvv/yyMp3rLDKiLoBmzJghCgsLlde9vb0iLS1NFBcXS6xq+GhtbRUAxOHDh4UQQnR0dAidTid27NihjDl9+rQAIMrKyoQQQuzZs0eo1WrhcrmUMZs3bxZGo1H4/f7ILkAE+Xw+MW7cOLF//37x7W9/WwkgrrPIiapDsJ6eHpSXlyMvL0+ZplarkZeXh7KyMomVDR8ejwfA/9ywW15ejkAgELbOsrKyYLfblXVWVlaGSZMmwWq1KmPy8/Ph9XpRU1MTweojq7CwEAsXLgxbNwDXWSRF1c2oV65cQW9vb9gPHQCsVivq6uokVTV8hEIhrF27FrNnz0Z2djYAwOVyQa/Xw2w2h421Wq1wuVzKmFut0/6+h1FJSQm++OILHD9+/Gt9XGeRE1UBRHdWWFiI6upqfP7557JLGdaam5vx8ssvY//+/YiJiZFdzogWVYdgFosFGo3ma1cj3G43bDabpKqGhzVr1mDXrl04ePAgxowZo0y32Wzo6elBR0dH2Pgb15nNZrvlOu3ve9iUl5ejtbUVjz/+OLRaLbRaLQ4fPox33nkHWq0WVquV6yxCoiqA9Ho9cnJyUFpaqkwLhUIoLS2Fw+GQWJk8QgisWbMGO3fuxIEDB5CZmRnWn5OTA51OF7bO6uvr4XQ6lXXmcDhQVVWF1tZWZcz+/fthNBoxYcKEyCxIBM2dOxdVVVWorKxU2rRp07B8+XLla66zCJF9FnygSkpKhMFgEO+//76ora0VL7zwgjCbzWFXI0aS1atXC5PJJA4dOiRaWlqUdv36dWXMqlWrhN1uFwcOHBAnTpwQDodDOBwOpb//kvK8efNEZWWl2Lt3r0hJSRlRl5RvvAomBNdZpERdAAkhxO9//3tht9uFXq8XM2bMEP/+979llyQNgFu2rVu3KmO6urrEiy++KEaNGiXi4uLE4sWLRUtLS9h8mpqaxPz580VsbKywWCxi/fr1IhAIRHhp5Lk5gLjOIoOP4yAiaaLqHBARPVwYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTT/H8q2bp+6TeUKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.5       -1.5       -5.        -5.        -3.1415927 -5.\n",
      " -0.        -0.       ], [1.5       1.5       5.        5.        3.1415927 5.        1.\n",
      " 1.       ], (8,), float32)\n",
      "env.action_space= Discrete(4)\n",
      "state= [-0.00166864  1.4159211  -0.16903177  0.22226675  0.00194033  0.03828825\n",
      "  0.          0.        ]\n",
      "action= 3\n",
      "next_state= [-0.00327435  1.4203348  -0.16087165  0.19616231  0.00224203  0.00603511\n",
      "  0.          0.        ]\n",
      "reward= 2.052907066271645\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f8e77f57400>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-178.7361532162933, 110.09158070393796)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de24ea3e3b4449519e427b145ac9e50d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/1.PPO.Lunar Lander')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(285.89778153869355, 18.83290154040063)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/1.PPO.Lunar Lander')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbYUlEQVR4nO3de3BU5d0H8O/ZKwnJbm5kl0BCglAQglwSLisVCwRCQikKXnCooqCAhr5eqPOSzog6YxtGrSJWU60V6diKorXtIDLQgDjaLZCYKNdw0ZgA2aQQsxvIPft7/0izrwuBZEOSJ4vfz8wzJuc8e/a3xz1fznmePRtNRARERAroVBdARD9cDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlJGWQC98sorSExMRL9+/TB58mTs27dPVSlEpIiSAHr33Xfx2GOP4cknn8QXX3yBsWPHIj09HZWVlSrKISJFNBU3o06ePBkTJ07E7373OwCA1+tFfHw8fvGLX2DNmjW9XQ4RKWLo7SdsbGxEQUEBsrOzfct0Oh3S0tLgdDrbfUxDQwMaGhp8v3u9XlRVVSE6OhqapvV4zUQUGBFBTU0N4uLioNNd/kKr1wPo7NmzaGlpgc1m81tus9lw9OjRdh+Tk5ODp59+ujfKI6JuVFZWhsGDB192fVDMgmVnZ8PtdvtaaWmp6pKIqBPCw8OvuL7Xz4BiYmKg1+tRUVHht7yiogJ2u73dx5jNZpjN5t4oj4i6UUdDJL1+BmQymZCSkoK8vDzfMq/Xi7y8PDgcjt4uh4gU6vUzIAB47LHHsGTJEqSmpmLSpElYv349Lly4gPvuu09FOUSkiJIAuvPOO/Gf//wHa9euhcvlwrhx47B9+/ZLBqaJ6Nqm5HNAV8vj8cBqtaoug4g64Ha7YbFYLrs+KGbBiOjaxAAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJQJOIA+/fRTzJs3D3FxcdA0DX/729/81osI1q5di4EDByIkJARpaWk4fvy4X5+qqiosXrwYFosFERERWLZsGc6fP39VL4SIgk/AAXThwgWMHTsWr7zySrvrn332WWzYsAG///3vsXfvXvTv3x/p6emor6/39Vm8eDEOHTqEnTt3YuvWrfj000+xfPnyrr8KIgpOchUAyIcffuj73ev1it1ul+eee863rLq6Wsxms7zzzjsiInL48GEBIPv37/f1+fjjj0XTNDl9+nSnntftdgsANja2Pt7cbvcVj+VuHQP65ptv4HK5kJaW5ltmtVoxefJkOJ1OAIDT6URERARSU1N9fdLS0qDT6bB37952t9vQ0ACPx+PXiCj4dWsAuVwuAIDNZvNbbrPZfOtcLhdiY2P91hsMBkRFRfn6XCwnJwdWq9XX4uPju7NsIlIkKGbBsrOz4Xa7fa2srEx1SUTUDbo1gOx2OwCgoqLCb3lFRYVvnd1uR2Vlpd/65uZmVFVV+fpczGw2w2Kx+DUiCn7dGkBJSUmw2+3Iy8vzLfN4PNi7dy8cDgcAwOFwoLq6GgUFBb4+u3btgtfrxeTJk7uzHCLq6wKY9BIRkZqaGiksLJTCwkIBIC+88IIUFhbKt99+KyIi69atk4iICPn73/8uX331lcyfP1+SkpKkrq7Ot405c+bI+PHjZe/evfLZZ5/J8OHD5a677up0DZwFY2MLjtbRLFjAAbR79+52n2jJkiUi0joV/8QTT4jNZhOz2SwzZ86U4uJiv22cO3dO7rrrLgkLCxOLxSL33Xef1NTUMIDY2K6x1lEAaSIiCDIejwdWq1V1GUTUAbfbfcUx26CYBSOiaxMDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJSJqAAysnJwcSJExEeHo7Y2FjccsstKC4u9utTX1+PrKwsREdHIywsDAsXLkRFRYVfn9LSUsydOxehoaGIjY3F448/jubm5qt/NUQUXCQA6enpsnHjRjl48KAUFRVJZmamJCQkyPnz5319Vq5cKfHx8ZKXlyf5+fkyZcoUufHGG33rm5ubJTk5WdLS0qSwsFC2bdsmMTExkp2d3ek63G63AGBjY+vjze12X/FYDiiALlZZWSkAZM+ePSIiUl1dLUajUbZs2eLrc+TIEQEgTqdTRES2bdsmOp1OXC6Xr09ubq5YLBZpaGjo1PMygNjYgqN1FEBXNQbkdrsBAFFRUQCAgoICNDU1IS0tzddn5MiRSEhIgNPpBAA4nU6MGTMGNpvN1yc9PR0ejweHDh1q93kaGhrg8Xj8GhEFvy4HkNfrxSOPPIKpU6ciOTkZAOByuWAymRAREeHX12azweVy+fp8P3za1reta09OTg6sVquvxcfHd7VsIupDuhxAWVlZOHjwIDZv3tyd9bQrOzsbbrfb18rKynr8OYmo5xm68qBVq1Zh69at+PTTTzF48GDfcrvdjsbGRlRXV/udBVVUVMBut/v67Nu3z297bbNkbX0uZjabYTabu1IqEfVlgQw6e71eycrKkri4ODl27Ngl69sGod9//33fsqNHjwpw6SB0RUWFr89rr70mFotF6uvrO1UHB6HZ2IKjdess2IMPPihWq1U++eQTKS8v97Xa2lpfn5UrV0pCQoLs2rVL8vPzxeFwiMPh8K1vm4afPXu2FBUVyfbt22XAgAGchmdjuwZbtwbQ5Z5k48aNvj51dXXy0EMPSWRkpISGhsqtt94q5eXlftspKSmRjIwMCQkJkZiYGFm9erU0NTUxgNjYrrHWUQBp/w2WoOLxeGC1WlWXQUQdcLvdsFgsl13Pe8GISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJluvR9QPTDo2nAggWAXg8UFgJuN3D2LOD1qq6sYzYbMG8ecPJka/vuO6CmRnVVBDCAqJM0DZg/Hxg5EmhqAjweoKQEOHWqNZBcrtaDu74eaGhQXa2/AQOABx4ARFprP3MGKC8HDh4Evv66tZ09C9TVAS0tqqvteQaDAddffz3uvvtulJSU4J133sF3332npBbeDU+dotMBb70FjBp16TqR1uD57jvgyBHgxRdbA6mvSE4G3nyz9TV8n0hr83iAqipg61bg7beD46yuK4xGI1JSUnD//ffjZz/7GWJiYtDc3Iz9+/fjueeew/bt21FfX9+tz9nR3fA8A6KAtZ1JeDxAZSVw+DBQWgp8+WXrgXzR36HsM9oCp7a29RLs+PHWur/6qvVMrrz82gyfkJAQ/PjHP8aKFSswa9YshIeHQ9M0AK2h5HA48M4772DHjh149tlnsXfv3l77Q6EMIOq0qio7tm1zIT+/NWSOHWs986mrU13Zlel04ThwQFBYeB7FxcC337aGTX19a5BeqywWC2bNmoUVK1Zg6tSpCAkJ8QXP92mahn79+mHevHm46aab8O677+Kll17CsWPH4O3pRO701xD2IfxGxN5ver1elixZoryOrrSUlBQZO3as8jp6q8XExMjSpUvF6XRKQ0ODeL3egI4vr9crp0+flieffFIGDhx4VbV09I2IPAOiHqFpOlz/o9mICIuHQDrsX3HuML4ucQKd6EuX0jQNgwYNwu23346lS5dixIgRMBgM7Z7xdGZbcXFxeOKJJ3D77bdj/fr1+OCDD1BdXQ3p5iFjBhD1iMjweNw07n9gsyR32FdEcPTMRygp3Q+v9xq+JuoBOp0OQ4cOxc9//nPcfffdSExMhO7i0fYu0uv1GD16NF599VXce++9+O1vf4uPPvoIjY2N3bJ9gAF0TdA0DRaLBWPGjEFjYyMKCwvRpHhwQ2cwoJ/RCp3W8VtMIAgxRcCgN6KRAdQpBoMBo0ePxrJly7Bw4ULY7fZuC56LGY1G3HjjjRg/fjw+/vhjvPDCCygoKEBDN3zeggEUpHQ6HaKiojBu3DhkZmZixowZGDZsGJqamvCPf/wDGzZsQFFREVoUfbDluvhpMBnCOtVX0zREW65DdORQlFce7OHKgpvJZEJqaipWrFiBzMxMREdHd+kyK1CapiE0NBQLFizA9OnT8cEHH+DZZ5/FyZMnr+qyjAEURPR6PWJjY+FwOJCRkYFp06YhMTERRqPR70149913Y968efjrX/+Kl19+GQcPHuz9INIERn1op7ub9GHQeGPQZfXv3x/Tpk3DihUrMGPGDISFhfVK8FxM0zRERUXh/vvvx5w5c/D6669j06ZNOH36dJdmzBhAfZzRaMTgwYMxdepUZGZmYurUqbDb7ZeEzvdpmobIyEgsXboU8+bNw3vvvYfc3FwUFxf3ShAZ9GYMHJAMg67zf07boOuHhIETccYVPGdAOp0Odrsd119/PTRNg9frRUtLC1paWvx+vvj3QH42Go2YNWsWli9fDofDcdmp9N6maRri4+Px1FNPYdGiRXj99dfx9ttvo6qqKqDtMIAuEhkZiTvvvBO1tbU4ceIEzpw5A7fbjZqaml77cJbZbEZSUhKmT5+OjIwMpKamwmazQdO0gN58mqYhNjYWWVlZuO222/DnP/8Zr732Gk6cONHtsxn+z6tDeH87tADuddbrTAgNieqxmrpTWFgYUlNTcccddyAjIwPx8fG+APJ6vRCRK/43kHV6vR6DBg3q8oxWT9Pr9Rg1ahSee+45LFiwAC+++CJ27dqFmk7ebMcA+i+dTocpU6Zg3bp1cDgcAICWlhacP38eLpcLLpcLJ0+exLFjx/D111+jrKzMF06NjY1XNejbdn09YsQIzJgxAxkZGRg3bhysVit0Ot1Vv/E0TYPdbsejjz6KRYsWYdOmTXjjjTdQUlLSrUFksdjR0HAeXm8LWqfTAwhL6GAy9odeb4TJGAqD0Yyamspuq+1qGY1GDB06FD/96U9x22234YYbbrjkbKSnBoH7Ok3TYDKZMG3aNKSmpmLnzp1Yv3499uzZ0+FjGUBo/cRoVlYWHn30UcTExPjeVAaDAWazGdHR0Rg9ejRmzpwJoDWYLly4gAsXLsDlcqGkpAQlJSU4fvw4Tpw4gVOnTuHcuXOorq5GU1NTuwe5pmkIDw9HcnIy0tPTMWvWLIwaNQoWi6XH/qXT6XQYNGgQ1qxZg8WLF+OPf/wjNm7ciFOnTl31tk3GUKQ5/hfQNOzZtx5al95aGq4bchMm37AUZ85+hV2fPw8RdfdGaJqG6Oho3HTTTVi0aBF+8pOfICYm5gcbNB3RNA39+/fH/PnzMWHCBAwZMqTDx/ygA0jTNEyYMAE5OTmYPn06DIbO7Q69Xg+LxQKLxYKBAwdi/PjxEBHfqXNtbS3Onj2LyspKfPPNNzh+/DhOnjyJb7/9FqdOncJ1112HOXPmYMaMGRg+fHivX9frdDoMGTIETz31FO6991784Q9/wJ/+9CecOXOmy9uMj0tFkn0aSlyf40JtYOMA31dX64Y1dBD6D4rF3v5v4vz5s13eVleFhIQgOTkZCxYswPz58zFs2LA+ewnUF2mahoiIiE71DSiAcnNzkZubi5KSEgDA6NGjsXbtWmRkZAAA6uvrsXr1amzevBkNDQ1IT0/Hq6++CpvN5ttGaWkpHnzwQezevRthYWFYsmQJcnJyOn3wd5ewsDAsXboUa9asgd1u75bLHE3ToNPpfOE0dOhQTJkyxRdMDQ0NaGhoQEhICMxms/I3dNuH2J555hksWbIEr732Gv7yl7+gsjKwSx9N0yFpsAMmfX8cK/0nmpsb0eytQ33zdwFtxytNqDh3FKfPFmHE4DmIHzgBR47vCGgbXdU21pKeno477rgDkyZN8rtpk3pGQEf94MGDsW7dOgwfPhwigk2bNmH+/PkoLCzE6NGj8eijj+Kjjz7Cli1bYLVasWrVKixYsACff/45gNZLl7lz58Jut+Nf//oXysvLcc8998BoNOI3v/lNj7zA9iQnJ+PXv/41MjMzodfre/xNpmka9Ho9QkNDERra+anp3qLX6zFixAg8//zzWLZsGV599VW89957OHfuXKceH2VNxI8S0lBTV47S0/vh9Tbj8PFt+Lr084DqOOf+Go1NtThWkodhcTPwo8TZOPb1brS09NyHEy0WC6ZMmYI77rgD6enpiIuLC3iwn65CQHeptSMyMlLeeOMNqa6uFqPRKFu2bPGtO3LkiAAQp9MpIiLbtm0TnU4nLpfL1yc3N1csFos0NDR0+jm7ejNqSEiILF++XEpLSwO+Qe+HpKmpSb744gtZtmyZREZGCnDlm1EnJN8l/3vfQfnp9F+LTqe/6pspw/rHytKFH8j/LP5coiKHXPX2Lr4Z1WQyyejRo+VXv/qV5OfnS21tLd8P3aztGO3oZtQuj6a1tLRg8+bNuHDhAhwOBwoKCtDU1IS0tDRfn5EjRyIhIQFOpxMA4HQ6MWbMGL9LsvT0dHg8Hhw6dKirpXTKsGHD8NZbb2HDhg2+aVNqn8FgwPjx45Gbm4vt27dj8eLFCA8Pb7ev2RSOHyXNQJO3Dse+/ed/Z8CuzoXaszhx6hMY9EYMst9w1dsD/v8jCYsWLcK7776LPXv24JlnnkFKSkqf+WzND1HAAy8HDhyAw+FAfX09wsLC8OGHH2LUqFEoKiqCyWS6ZPDJZrPB9d+vx3O5XH7h07a+bd3ltI2dtPF4PJ2u12w24/bbb8fTTz+NpKQkvtECYDQaMXHiRGzcuBH5+fnYsWMH7rnnHr8+Os2AxEGCqKgqzAwZhym18d3y3FGRkbDHluLHIddh/MR7On7AFYSHh2PcuHG4+eabkZSU1CuX3dQ5AQfQiBEjUFRUBLfbjffffx9Llizp1Hz/1cjJycHTTz8d8OMSExOxdu1aLFq0CCEhIT1Q2bVP0zTft+a1fT7q8mb3QAULe2Cb1FcEfAlmMpkwbNgwpKSkICcnB2PHjsVLL70Eu92OxsZGVFdX+/WvqKiA3W4HANjtdlRc9H2dbb+39WlPdnY23G63r5WVlV2xRqPRiIULF+Ljjz/Gvffey/Ah6qOu+hNVbdPLKSkpMBqNyMvL860rLi5GaWmp719Oh8OBAwcO+E3z7ty5ExaLBaPa+7bz/zKbzb6p7bZ2OXFxcVi/fj02bdqEkSNH8lSbqC8LZGR7zZo1smfPHvnmm2/kq6++kjVr1oimabJjxw4REVm5cqUkJCTIrl27JD8/XxwOhzgcDt/jm5ubJTk5WWbPni1FRUWyfft2GTBggGRnZ3dphP37zWAwSGZmpnz55ZfS0tLCWQ0ihTo7CxZQAC1dulSGDBkiJpNJBgwYIDNnzvSFj4hIXV2dPPTQQxIZGSmhoaFy6623Snl5ud82SkpKJCMjQ0JCQiQmJkZWr14tTU1NgZRxSQDFxsbK888/L263m8FD1Ad0NoCC+u+C6XQ63HzzzVi3bh1SU1N5jw5RH9F2jF7TfxcsOzsbv/zlL2G1WjnWQxSEgvoMqKqqCpGRkarLIaKLdPYMKKivWfR6veoSiOgqBHUAEVFwYwARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMoYVBfQFSICAPB4PIorIaL2tB2bbcfq5QRlAJ07dw4AEB8fr7gSIrqSmpoaWK3Wy64PygCKiooCAJSWll7xxZE/j8eD+Ph4lJWVwWKxqC4nKHCfdY2IoKamBnFxcVfsF5QBpNO1Dl1ZrVa+KbrAYrFwvwWI+yxwnTk54CA0ESnDACIiZYIygMxmM5588kmYzWbVpQQV7rfAcZ/1LE06micjIuohQXkGRETXBgYQESnDACIiZRhARKRMUAbQK6+8gsTERPTr1w+TJ0/Gvn37VJekTE5ODiZOnIjw8HDExsbilltuQXFxsV+f+vp6ZGVlITo6GmFhYVi4cCEqKir8+pSWlmLu3LkIDQ1FbGwsHn/8cTQ3N/fmS1Fm3bp10DQNjzzyiG8Z91kvkSCzefNmMZlM8uabb8qhQ4fkgQcekIiICKmoqFBdmhLp6emyceNGOXjwoBQVFUlmZqYkJCTI+fPnfX1Wrlwp8fHxkpeXJ/n5+TJlyhS58cYbfeubm5slOTlZ0tLSpLCwULZt2yYxMTGSnZ2t4iX1qn379kliYqLccMMN8vDDD/uWc5/1jqALoEmTJklWVpbv95aWFomLi5OcnByFVfUdlZWVAkD27NkjIiLV1dViNBply5Ytvj5HjhwRAOJ0OkVEZNu2baLT6cTlcvn65ObmisVikYaGht59Ab2opqZGhg8fLjt37pSbb77ZF0DcZ70nqC7BGhsbUVBQgLS0NN8ynU6HtLQ0OJ1OhZX1HW63G8D/37BbUFCApqYmv302cuRIJCQk+PaZ0+nEmDFjYLPZfH3S09Ph8Xhw6NChXqy+d2VlZWHu3Ll++wbgPutNQXUz6tmzZ9HS0uL3Px0AbDYbjh49qqiqvsPr9eKRRx7B1KlTkZycDABwuVwwmUyIiIjw62uz2eByuXx92tunbeuuRZs3b8YXX3yB/fv3X7KO+6z3BFUA0ZVlZWXh4MGD+Oyzz1SX0qeVlZXh4Ycfxs6dO9GvXz/V5fygBdUlWExMDPR6/SWzERUVFbDb7Yqq6htWrVqFrVu3Yvfu3Rg8eLBvud1uR2NjI6qrq/36f3+f2e32dvdp27prTUFBASorKzFhwgQYDAYYDAbs2bMHGzZsgMFggM1m4z7rJUEVQCaTCSkpKcjLy/Mt83q9yMvLg8PhUFiZOiKCVatW4cMPP8SuXbuQlJTktz4lJQVGo9FvnxUXF6O0tNS3zxwOBw4cOIDKykpfn507d8JisWDUqFG980J60cyZM3HgwAEUFRX5WmpqKhYvXuz7mfusl6geBQ/U5s2bxWw2y1tvvSWHDx+W5cuXS0REhN9sxA/Jgw8+KFarVT755BMpLy/3tdraWl+flStXSkJCguzatUvy8/PF4XCIw+HwrW+bUp49e7YUFRXJ9u3bZcCAAT+oKeXvz4KJcJ/1lqALIBGRl19+WRISEsRkMsmkSZPk3//+t+qSlAHQbtu4caOvT11dnTz00EMSGRkpoaGhcuutt0p5ebnfdkpKSiQjI0NCQkIkJiZGVq9eLU1NTb38atS5OIC4z3oHv46DiJQJqjEgIrq2MICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJl/g8DWZB1YyITwgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "289.82857258297884 164 [0.17620373294079286, 0.16220330258954732, 0.0809007627084111, 0.5073526572152229, -0.499818711383341, 0.631813851261428, -0.3545947211171665, -1.0070048348956107, -1.2064146996285956, -1.2169994902541816, -0.6563874240848679, -0.9358495901069819, -1.3030842499314577, -1.6067412620130017, -1.7970015642710837, -2.254420794451903, -2.231539032546749, -2.205789552557718, -2.5365759946691626, -2.282116613951216, -1.4149922810397288, -1.9876987395138588, -1.3249989071920811, -1.7659053020176145, 1.666890153202172, 1.9893991781921556, -1.8317427412715404, -1.7935768317087764, 3.718253556270821, 4.812593414717679, 1.8010588588381438, -2.030756017884015, 0.7638008318559855, 1.5398837337343252, 0.5563416956563685, 1.504666611317259, 0.9328736743401123, 2.4066371697561637, 2.4339757917254135, 3.7253142136588737, 0.6640342835368187, -1.8912184377620906, 3.937421031608136, 1.7546994826778757, -1.8092980439585336, -1.5488961155926677, 1.4564751575926038, 3.7991907734726285, -1.375493735397838, 5.193769928679058, 0.9691473010023628, 4.484237967642753, -1.07618119499861, 3.7449542848270996, 2.9040933507245486, -0.7645384379238738, -0.5343637825205565, -0.1915564981871671, 1.5491578669855073, 4.101642926657843, 4.870058163939302, 4.60152532641784, 2.4592290019183, -0.18548547891506814, 4.26667861984559, 3.6389041308888865, 0.11703002861705272, 2.8872986734797736, 2.5353989862654887, 3.2720754397344196, 0.5436502174018483, 2.4197229592620717, 0.8921832891921053, 3.2944280772602896, 3.2440329136120427, 1.575735596435291, 4.11394246313341, 3.8819899635459025, 3.5458637286318124, -0.7115064339839126, 4.7951001416482315, 5.29045990110225, 3.561281808350512, 2.1013568632275907, 0.11453567963613523, 4.077946209837887, 0.3655229559367268, -0.7762954879536028, 5.145385390211072, 0.1878170471558616, 4.719804128252631, -1.0758831283211368, -1.0084278712284191, 3.410120466534477, 4.585339091341933, -1.4793273806360634, 1.298222129030745, 2.2316751627464386, 0.4444026427729398, 0.9833145913791952, -1.59007905117103, 3.761634558558609, 2.9070058368535756, 0.6025865409708444, 2.8344166243136444, 2.0237026522467874, 1.9313969166816463, -0.437151427836919, 2.1580778273729466, 1.5076603061695166, 11.679729275874049, 4.063535019699015, 3.0566637266701178, 14.41883692791293, 3.6649399141126304, 1.9620256761917865, 1.0906909200810102, 2.460982053817735, 2.2681938786666342, 0.8950406726262621, 0.4056605930238817, 1.192246572956763, 2.6824677542378033, 1.5483633530284493, 1.3169207764915178, 0.9587819756643157, 0.6966730182791672, 0.6325270778729486, 0.7030034573383013, 0.7842098941212132, 0.7976223757972036, 0.75315560524494, 0.7160207129422336, 0.6842316168979892, 0.6014157921158869, 0.5552146935315285, 1.1757694033049013, 0.5694298615085458, 0.5732318599538004, 0.41948490959509677, -0.13612610990629292, 0.0698970788219686, 0.09022003887443653, -0.017145188557561752, 0.007025564501496717, 0.01570116668681898, 0.0005982789991794135, -0.00020920799096479925, 0.0024772312676883956, 0.0008528925265451903, -0.00024343285788397395, 0.00037058356794617, 0.00028048134481828413, -9.24722426152158e-05, 4.4773093414107734e-05, 5.8161844370019367e-05, -8.25846479379777e-06, 3.301119603449365e-06, 8.71867096918777e-06, 3.694757197081344e-06, -3.324168753593426e-06, -8.715796262492859e-08, 5.942669307756887e-07, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
